package com.gxljc.bear.crawler.itaogao.stcn;

import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;

import org.apache.log4j.Logger;

import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;
import com.gxljc.bear.crawler.itaogao.NewsBaseCrawler;
import com.gxljc.bear.crawler.itaogao.NewsBaseTable;
import com.gxljc.bear.crawler.util.DateUtil;
import com.gxljc.bear.crawler.util.JsoupUtil;

/**
 * 证券时报
 * 
 * @author Administrator->haitao
 * @since 2021年9月14日 下午5:51:54
 */
public class STCNDetailCrawler extends NewsBaseCrawler {

	private static final long serialVersionUID = -1084336561181882139L;
	final static Logger LOG = Logger.getLogger(STCNDetailCrawler.class);
	final static String LIST_DEFAULT = "http://epaper.stcn.com/paper/zqsb/html/2016-09/14/node_2.htm";
	final static String LIST_FORMAT = "http://epaper.stcn.com/paper/zqsb/html/%s/node_2.htm";

	final static String URL_FORMAT_DEFAULT = "http://epaper.stcn.com/paper/zqsb/html/2016-09/13/content_876133.htm";
	final static String URL_FORMAT = "http://epaper.stcn.com/paper/zqsb/html/%s/%s";

	public STCNDetailCrawler() {
		super("t_newspaper_stcn", DolphinCrawlerConsts.MongoDBName.ITAOGAO.getName());
	}

	@Override
	protected List<String> genSeeds() {
		LOG.info("--- genSeed of stcn ----");
		List<String> seeds = new ArrayList<>();
		try {
			for (String date : genDatesTillNow(DEFAULT_DATE_FORMAT)) {
				// fetch url list by parser
				String url = String.format(LIST_FORMAT, date);
				byte[] bytes = fetch(url, proxy);
				if (bytes == null) {
					continue;
				}
				String html = new String(bytes);
				List<String> hrefList = JsoupUtil.extractAttrList(html, ".news_list li a", "href");
				List<String> urls = new ArrayList<>();
				for (String s : hrefList) {
					urls.add(String.format(URL_FORMAT, date, s));
				}
				seeds.addAll(urls);
			}

		} catch (Exception e) {
			e.printStackTrace();
		}
		LOG.info("total size=" + seeds.size());
		return seeds;
	}

	@Override
	protected List<String> genNewSeeds() {
		LOG.info("--- genSeed of stcn ----");
		List<String> seeds = new ArrayList<>();
		String date = DateUtil.dateFormat(new Date(), "yyyy-MM/dd");
		// fetch url list by parser
		String url = String.format(LIST_FORMAT, date);
		String html = "";
		try {
			byte[] bytes = fetch(url, proxy);
			if (bytes == null) {
				LOG.error(String.format("fetch empty for url=", url));
				return seeds;
			}
			html = new String(bytes);
			List<String> hrefList = JsoupUtil.extractAttrList(html, ".news_list li a", "href");
			if (hrefList.size() == 0) {
				hrefList = JsoupUtil.extractAttrList(html, "div.area ul li a", "href");
			}
			List<String> urls = new ArrayList<>();
			for (String s : hrefList) {
				urls.add(String.format(URL_FORMAT, date, s));
			}
			seeds.addAll(urls);
		} catch (Exception e) {
			e.printStackTrace();
		}

		LOG.info("total size=" + seeds.size());
		return seeds;
	}

	@Override
	protected Pair<NewsBaseTable, byte[]> genTable(String url) {

		NewsBaseTable table = new NewsBaseTable();
		// parse page by Jsoup
		byte[] html = null;
		try {
			html = fetch(url, proxy);
		} catch (Exception e) {
			e.printStackTrace();
		}
		if (html != null) {
			String htmlPage = new String(html);
			table = extractTable(url, htmlPage);
		}
		// table.setPublishDate(extractDate(url));
		// set default value
		table.setId(StringUtil.genMD5Val(url));
		table.setCrawlUrl(url);
		table.setCrawlDate(new Date());
		table.setCrawlFlag(1);
		return new Pair<>(table, html);

	}

	private NewsBaseTable extractTable(String url, String html) {
		NewsBaseTable table = new NewsBaseTable();
		// fetch source
		String css0 = ".time span";
		List<String> box = JsoupUtil.extractList(html, css0);
		LOG.info("box=" + box);
		if (validateObject(box)) {
			table.setPublishDate(DateUtil.parseFormat(box.get(0), "yyyy-MM-dd"));
			table.setSource(extractByPrefix("来源：", box.get(1)));
		}
		// fetch author
		String css1 = ".time";
		List<String> authorBox = JsoupUtil.extractList(html, css1);
		if (validateObject(authorBox)) {
			table.setAuthor(extractByPrefix("作者：", authorBox.get(0)));
		}
		// fetch title
		String css2 = ".tc_tit p";
		List<String> title = JsoupUtil.extractList(html, css2);
		if (validateObject(title)) {
			table.setTitle(title.get(0));
		}
		// fetch images
		String css3 = ".tc_con table tbody tr td img";
		List<String> images = JsoupUtil.extractAttrList(html, css3, "src");
		table = processImages(url, table, images);
		// fetch content
		String css4 = ".tc_con founder-content p";
		List<String> contents = JsoupUtil.extractList(html, css4);
		List<String> nContents = new LinkedList<>();
		for (String s : contents) {
			if ("&nbsp;".equals(s) || s.isEmpty()) {
				continue;
			}
			nContents.add(s.replaceAll("&nbsp;", ""));
		}
		if (validateObjects(nContents)) {
			table.setContent(genContent(nContents, table.getImages()));
		}
		return table;
	}

}
